'''
Date: 2024-05-22 10:08:36
LastEditors: zhaozhenguo zhaozhenguo@tuyoogame.com
LastEditTime: 2024-05-23 10:57:45
'''
import os
import json

def process_file(input_path, output_path, max_lines):
    seen_cards_list = set()
    unique_lines = []

    with open(input_path, 'r') as infile:
        for line in infile:
            if len(unique_lines) >= max_lines:
                break
            try:
                data = json.loads(line.strip())  
                cards_list = tuple(tuple(x) for x in data.get('cardsList', [])) 
                if cards_list not in seen_cards_list:
                    seen_cards_list.add(cards_list)
                    unique_lines.append(line)
            except ValueError:  
                print("Skipping invalid JSON line in file: {}".format(input_path))
    
    output_dir = os.path.dirname(output_path)
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    
    with open(output_path, 'w') as outfile:
        outfile.writelines(unique_lines)

def process_folder(input_folder, output_folder):
    for root, dirs, files in os.walk(input_folder):
        for filename in files:
            if filename.endswith('.json'):
                input_path = os.path.join(root, filename)
                relative_path = os.path.relpath(input_path, input_folder)
                output_path = os.path.join(output_folder, relative_path)
                print("Processing {}".format(input_path))
                
                max_lines = 90000 
                if any(keyword in relative_path for keyword in ["crest", "trough", "pool_2"]):
                    max_lines = 90000
                elif "pool_1" in relative_path:
                    max_lines = 90000
                
                process_file(input_path, output_path, max_lines)

# Example usage
# input_folder = './dist/prod_1/crest'  
# output_folder = './dist/prod_1/crest_rm'  
# process_folder(input_folder, output_folder)
input_folder = './dist/prod_1/trough'  
output_folder = './dist/prod_1/trough_rm'  
process_folder(input_folder, output_folder)